{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "plaintext"
    }
   },
   "outputs": [],
   "source": [
    "# 导入数据集\n",
    "LENTH = 200\n",
    "\n",
    "# 读取数据集\n",
    "import pandas as pd\n",
    "\n",
    "file_path = \"hallucinated_answers.csv\"\n",
    "\n",
    "# 使用 pandas 读取 CSV 文件\n",
    "df = pd.read_csv(file_path)\n",
    "\n",
    "# 选择第一列（索引 0）、第四列（索引 3）和第六列（索引 5）\n",
    "df_selected = df.iloc[:LENTH, [0, 3, 5]]\n",
    "# 显示读取的数据\n",
    "print(df_selected.head())  # 打印前几行以确认数据是否成功读取"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {},
   "outputs": [],
   "source": [
    "# 一个问题生成一个答案\n",
    "def ask_question(clientname,client,question):#选手、问题\n",
    "  # try:\n",
    "  if clinetname == \"doubi\":\n",
    "    response = client.chat.completions.create(\n",
    "        model=\"doubao-1-5-vision-pro-32k-250115\",\n",
    "        messages=[\n",
    "            {\n",
    "                \"role\": \"user\",\n",
    "                \"content\": [\n",
    "                    {\"type\": \"text\", \"text\": f\"{question}\"},#这里是不是可以提示回答的长短？\n",
    "                ],\n",
    "            }\n",
    "        ],\n",
    "    )\n",
    "    return response\n",
    "\n",
    "#   elif clinetname == \"gemini\":# 最好先别测这个，访问太少不好搞\n",
    "#     client = genai.Client(api_key=\"AIzaSyB2Atc4agblDdsqa3_xKh3_K81tr2OW0t8\")\n",
    "#     response = client.models.generate_content(\n",
    "#         model=\"gemini-2.0-flash\",\n",
    "#         contents=f'{question}',\n",
    "#     )\n",
    "#     return response.text\n",
    "\n",
    "  elif clinetname == \"deepseek\":\n",
    "    completion = client.chat.completions.create(\n",
    "    model = \"moonshot-v1-8k\",\n",
    "    messages = [\n",
    "        {\"role\": \"system\", \"content\": \"我下面会给你一个问题\"},\n",
    "        {\"role\": \"user\", \"content\": f\"{question}\"}\n",
    "    ],\n",
    "    temperature = 0.3,\n",
    "    )\n",
    "    return completion.choices[0].message.content\n",
    "\n",
    "  elif clinetname == \"kimi\":\n",
    "\n",
    "    completion = client.chat.completions.create(\n",
    "        model = \"moonshot-v1-8k\",\n",
    "        messages = [\n",
    "            {\"role\": \"system\", \"content\": \"我下面会给你一个问题\"},\n",
    "            {\"role\": \"user\", \"content\": f\"{question}\"}\n",
    "        ],\n",
    "        temperature = 0.3,\n",
    "    )\n",
    "    return completion.choices[0].message.content\n",
    "\n",
    "  elif clinetname == \"qwen\":\n",
    "    completion = client.chat.completions.create(\n",
    "      model=\"qwen-plus\", # 此处以qwen-plus为例，可按需更换模型名称。模型列表：https://help.aliyun.com/zh/model-studio/getting-started/models\n",
    "      messages=[\n",
    "          {\"role\": \"system\", \"content\": \"我下面会给你一个问题\"},\n",
    "          {\"role\": \"user\", \"content\": f\"{question}\"}]\n",
    "      )\n",
    "\n",
    "    return completion.choices[0].message.content"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "plaintext"
    }
   },
   "outputs": [],
   "source": [
    "def all_response(name,client,questions):\n",
    "  responses = list()\n",
    "  for question in questions:\n",
    "    response = ask_question(name,client,question)\n",
    "    responses.append(response)\n",
    "    print(f\"问题为{question}，答案为{response}\")\n",
    "\n",
    "  # 打开文件，按换行符分割写入\n",
    "  with open(\"response1.txt\", \"w\", encoding=\"utf-8\") as f:\n",
    "    for item in responses:\n",
    "        f.write(f\"{item}\\n\")  # 每个元素后添加换行符 \\n\n",
    "  return responses # 这里索引和questins对应\n",
    "# 一个问题生成一个答案\n",
    "def ask_question(clinetname,client,question):#模型名字、选手、问题\n",
    "  # try:\n",
    "  if clinetname == \"doubi\":\n",
    "    response = client.chat.completions.create(\n",
    "        model=\"doubao-1-5-vision-pro-32k-250115\",\n",
    "        messages=[\n",
    "            {\n",
    "                \"role\": \"user\",\n",
    "                \"content\": [\n",
    "                    {\"type\": \"text\", \"text\": f\"{question}\"},#这里是不是可以提示回答的长短？\n",
    "                ],\n",
    "            }\n",
    "        ],\n",
    "    )\n",
    "    return response\n",
    "\n",
    "  elif clinetname == \"gemini\":# 最好先别测这个，访问太少不好搞\n",
    "    client = genai.Client(api_key=\"AIzaSyB2Atc4agblDdsqa3_xKh3_K81tr2OW0t8\")\n",
    "    response = client.models.generate_content(\n",
    "        model=\"gemini-2.0-flash\",\n",
    "        contents=f'{question}',\n",
    "    )\n",
    "    return response.text\n",
    "\n",
    "  elif clinetname == \"deepseek\":\n",
    "    completion = client.chat.completions.create(\n",
    "    model = \"deepseek-chat\",\n",
    "    messages = [\n",
    "        {\"role\": \"system\", \"content\": \"我下面会给你一个问题\"},\n",
    "        {\"role\": \"user\", \"content\": f\"{question}\"}\n",
    "    ],\n",
    "    temperature = 0.3,\n",
    "    )\n",
    "    return completion.choices[0].message.content\n",
    "\n",
    "  elif clinetname == \"kimi\":\n",
    "\n",
    "    completion = client.chat.completions.create(\n",
    "        model = \"moonshot-v1-8k\",\n",
    "        messages = [\n",
    "            {\"role\": \"system\", \"content\": \"我下面会给你一个问题\"},\n",
    "            {\"role\": \"user\", \"content\": f\"{question}\"}\n",
    "        ],\n",
    "        temperature = 0.3,\n",
    "    )\n",
    "    return completion.choices[0].message.content\n",
    "\n",
    "  elif clinetname == \"qwen\":\n",
    "    completion = client.chat.completions.create(\n",
    "      model=\"qwen-plus\", # 此处以qwen-plus为例，可按需更换模型名称。模型列表：https://help.aliyun.com/zh/model-studio/getting-started/models\n",
    "      messages=[\n",
    "          {\"role\": \"system\", \"content\": \"我下面会给你一个问题\"},\n",
    "          {\"role\": \"user\", \"content\": f\"{question}\"}]\n",
    "      )\n",
    "\n",
    "    return completion.choices[0].message.content\n",
    "裁判判决\n",
    "def judge(clinetname,client,question,answer):#模型名字、裁判、问题\n",
    "  # try:\n",
    "  if clinetname == \"doubi\":\n",
    "    response = client.chat.completions.create(\n",
    "        model=\"doubao-1-5-vision-pro-32k-250115\",\n",
    "        messages=[\n",
    "            {\n",
    "                \"role\": \"user\",\n",
    "                \"content\": [\n",
    "                    {\"type\": \"text\", \"text\": f'我下面会给你一个问题以及一个别的大语言模型给出的答案，请你判断这个回答是否出现幻觉现象，只需要回答是或不是,不要有多余的字符，只需要是，或者不是。问题为：{question}；答案为：{answer}'},#这里是不是可以提示回答的长短？\n",
    "                ],\n",
    "            }\n",
    "        ],\n",
    "    )\n",
    "    return response.choices[0].message.content.strip()\n",
    "\n",
    "  elif clinetname == \"gemini\":# 最好先别测这个，访问太少不好搞\n",
    "    client = genai.Client(api_key=\"AIzaSyB2Atc4agblDdsqa3_xKh3_K81tr2OW0t8\")\n",
    "    response = client.models.generate_content(\n",
    "        model=\"gemini-2.0-flash\",\n",
    "        contents=f'我下面会给你一个问题以及一个别的大语言模型给出的答案，请你判断这个回答是否出现幻觉现象，只需要回答是或不是,不要有多余的字符，只需要是，或者不是。问题为：{question}；答案为：{answer}',\n",
    "    )\n",
    "    return response.text.strip()\n",
    "\n",
    "  elif clinetname == \"deepseek\":\n",
    "    response = client.chat.completions.create(\n",
    "    model=\"deepseek-chat\",\n",
    "    messages=[\n",
    "     {\"role\": \"system\", \"content\": \"我下面会给你一个问题以及一个别的大语言模型给出的答案，请你判断这个回答是否出现幻觉现象，只需要回答是或不是,不要有多余的字符，只需要是，或者不是\"},\n",
    "      {\"role\": \"user\", \"content\": f\"问题为：{question}；答案为：{answer}\"}\n",
    "    ],\n",
    "    stream=False\n",
    "    )\n",
    "    return response.choices[0].message.content.strip()\n",
    "\n",
    "\n",
    "  elif clinetname == \"kimi\":\n",
    "\n",
    "    completion = client.chat.completions.create(\n",
    "        model = \"moonshot-v1-8k\",\n",
    "        messages = [\n",
    "          {\"role\": \"system\", \"content\": \"我下面会给你一个问题以及一个别的大语言模型给出的答案，请你判断这个回答是否出现幻觉现象，只需要回答是或不是,不要有多余的字符，只需要是，或者不是\"},\n",
    "          {\"role\": \"user\", \"content\": f\"问题为：{question}；答案为：{answer}\"}\n",
    "        ],\n",
    "        temperature = 0.3,\n",
    "    )\n",
    "    return completion.choices[0].message.content.strip()\n",
    "\n",
    "  elif clinetname == \"qwen\":\n",
    "    completion = client.chat.completions.create(\n",
    "      model=\"qwen-plus\", # 此处以qwen-plus为例，可按需更换模型名称。模型列表：https://help.aliyun.com/zh/model-studio/getting-started/models\n",
    "      messages=[\n",
    "          {\"role\": \"system\", \"content\": \"我下面会给你一个问题以及一个别的大语言模型给出的答案，请你判断这个回答是否出现幻觉现象，只需要回答是或不是,不要有多余的字符，只需要是，或者不是\"},\n",
    "          {\"role\": \"user\", \"content\": f\"问题为：{question}；答案为：{answer}\"}\n",
    "          ]\n",
    "      )\n",
    "\n",
    "    return completion.choices[0].message.content.strip()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "裁判判决"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "plaintext"
    }
   },
   "outputs": [],
   "source": [
    "def judge(clinetname,client,question,answer):#模型名字、裁判、问题\n",
    "  # try:\n",
    "  if clinetname == \"doubi\":\n",
    "    response = client.chat.completions.create(\n",
    "        model=\"doubao-1-5-vision-pro-32k-250115\",\n",
    "        messages=[\n",
    "            {\n",
    "                \"role\": \"user\",\n",
    "                \"content\": [\n",
    "                    {\"type\": \"text\", \"text\": f'我下面会给你一个问题以及一个别的大语言模型给出的答案，请你判断这个回答是否出现幻觉现象，只需要回答是或不是,不要有多余的字符，只需要是，或者不是。问题为：{question}；答案为：{answer}'},#这里是不是可以提示回答的长短？\n",
    "                ],\n",
    "            }\n",
    "        ],\n",
    "    )\n",
    "    return response.choices[0].message.content.strip()\n",
    "\n",
    "  elif clinetname == \"gemini\":# 最好先别测这个，访问太少不好搞\n",
    "    client = genai.Client(api_key=\"AIzaSyB2Atc4agblDdsqa3_xKh3_K81tr2OW0t8\")\n",
    "    response = client.models.generate_content(\n",
    "        model=\"gemini-2.0-flash\",\n",
    "        contents=f'我下面会给你一个问题以及一个别的大语言模型给出的答案，请你判断这个回答是否出现幻觉现象，只需要回答是或不是,不要有多余的字符，只需要是，或者不是。问题为：{question}；答案为：{answer}',\n",
    "    )\n",
    "    return response.text.strip()\n",
    "\n",
    "  elif clinetname == \"deepseek\":\n",
    "    response = client.chat.completions.create(\n",
    "    model=\"deepseek-chat\",\n",
    "    messages=[\n",
    "     {\"role\": \"system\", \"content\": \"我下面会给你一个问题以及一个别的大语言模型给出的答案，请你判断这个回答是否出现幻觉现象，只需要回答是或不是,不要有多余的字符，只需要是，或者不是\"},\n",
    "      {\"role\": \"user\", \"content\": f\"问题为：{question}；答案为：{answer}\"}\n",
    "    ],\n",
    "    stream=False\n",
    "    )\n",
    "    return response.choices[0].message.content.strip()\n",
    "\n",
    "\n",
    "  elif clinetname == \"kimi\":\n",
    "\n",
    "    completion = client.chat.completions.create(\n",
    "        model = \"moonshot-v1-8k\",\n",
    "        messages = [\n",
    "          {\"role\": \"system\", \"content\": \"我下面会给你一个问题以及一个别的大语言模型给出的答案，请你判断这个回答是否出现幻觉现象，只需要回答是或不是,不要有多余的字符，只需要是，或者不是\"},\n",
    "          {\"role\": \"user\", \"content\": f\"问题为：{question}；答案为：{answer}\"}\n",
    "        ],\n",
    "        temperature = 0.3,\n",
    "    )\n",
    "    return completion.choices[0].message.content.strip()\n",
    "\n",
    "  elif clinetname == \"qwen\":\n",
    "    completion = client.chat.completions.create(\n",
    "      model=\"qwen-plus\", # 此处以qwen-plus为例，可按需更换模型名称。模型列表：https://help.aliyun.com/zh/model-studio/getting-started/models\n",
    "      messages=[\n",
    "          {\"role\": \"system\", \"content\": \"我下面会给你一个问题以及一个别的大语言模型给出的答案，请你判断这个回答是否出现幻觉现象，只需要回答是或不是,不要有多余的字符，只需要是，或者不是\"},\n",
    "          {\"role\": \"user\", \"content\": f\"问题为：{question}；答案为：{answer}\"}\n",
    "          ]\n",
    "      )\n",
    "\n",
    "    return completion.choices[0].message.content.strip()"
   ]
  },
  {
   "cell_type": "markdown",
   "metadata": {},
   "source": [
    "# 获取答案"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "plaintext"
    }
   },
   "outputs": [],
   "source": [
    "from openai import OpenAI\n",
    "#deepseek\n",
    "deepseek = OpenAI(api_key=\"sk-345d4d96bd464da2bfebb09bc43b4aa4\", base_url=\"https://api.deepseek.com\")\n",
    "\n",
    "# 设立问题集合\n",
    "questions = df['Question'][200:LENTH+200]\n",
    "# 获取答案\n",
    "responses = all_response('deepseek',deepseek,questions)\n",
    "\n",
    "import json\n",
    "# 将 list 写入 JSON 文件\n",
    "with open('ds.json', 'w') as json_file:\n",
    "    json.dump(responses, json_file)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "plaintext"
    }
   },
   "outputs": [],
   "source": [
    "import json\n",
    "import pandas as pd\n",
    "\n",
    "# 假设你的文件名是 'file.json'\n",
    "with open('ds.json', 'r') as file:\n",
    "    data = json.load(file)\n",
    "\n",
    "# 转换为 DataFrame\n",
    "responses = pd.DataFrame(data)\n",
    "responses = responses[0]\n",
    "\n",
    "# 设立问题集合\n",
    "questions = df['Question'][200:LENTH+200]\n",
    "questions = pd.Series(questions.values, index=range(0, len(questions)))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": null,
   "metadata": {
    "vscode": {
     "languageId": "plaintext"
    }
   },
   "outputs": [],
   "source": [
    "from openai import OpenAI\n",
    "from google import genai\n",
    "\n",
    "# 建立clinet\n",
    "# gemini：\n",
    "# gemini = genai.Client(api_key=\"AIzaSyB2Atc4agblDdsqa3_xKh3_K81tr2OW0t8\")\n",
    "\n",
    "#deepseek\n",
    "deepseek = OpenAI(api_key=\"sk-345d4d96bd464da2bfebb09bc43b4aa4\", base_url=\"https://api.deepseek.com\")\n",
    "\n",
    "#qwen\n",
    "qwen = OpenAI(api_key=\"sk-fb3964c128a74650be4c226d9b7dd9bc\",base_url=\"https://dashscope.aliyuncs.com/compatible-mode/v1\",)\n",
    "\n",
    "#doubi\n",
    "doubi = OpenAI(\n",
    "    base_url=\"https://ark.cn-beijing.volces.com/api/v3\",\n",
    "    api_key=\"01e37152-1149-4888-8c3c-0fb25cf40c4b\"\n",
    ")\n",
    "\n",
    "\n",
    "judgelist1 = list()\n",
    "judgelist2 = list()\n",
    "\n",
    "client1_accurary = 0.8\n",
    "client2_accurary = 0.7\n",
    "# 最终分数\n",
    "socre =0\n",
    "\n",
    "for i in range(1,LENTH):\n",
    "  judge1 = judge('qwen',qwen,questions[i],responses[i])\n",
    "  print(f\"千问认为{judge1}\")\n",
    "  judge2 = judge('deepseek',deepseek,questions[i],responses[i])\n",
    "  print(f\"deepseek认为{judge2}\")\n",
    "  judgelist1.append(judge1)\n",
    "  judgelist2.append(judge2)\n",
    "\n",
    "  # 结合正确率给分\n",
    "  if judge1 == \"是\" and judge2 == \"是\":\n",
    "      print(\"ds认为 是 ，gemini认为 是 \")\n",
    "      socre = socre + client1_accurary + client2_accurary -client2_accurary*client1_accurary\n",
    "  elif judge1 == \"不是\" and judge2 == \"是\":\n",
    "      print(\"ds认为 不是 ，gemini认为 是 \")\n",
    "      socre = socre + (1-client1_accurary)*client2_accurary\n",
    "  elif judge1 == \"是\" and judge2 == \"不是\":\n",
    "      print(\"ds认为 是 ，gemini认为 不是 \")\n",
    "      socre = socre +  (1-client2_accurary)*client1_accurary\n",
    "  elif judge1 == \"不是\" and judge2 == \"不是\":\n",
    "      print(\"ds认为 不是 ，gemini认为 不是\")\n",
    "      socre = socre +  (1-client1_accurary)*(1-client2_accurary)\n",
    "\n",
    "print(f'分数为{socre}')\n"
   ]
  }
 ],
 "metadata": {
  "language_info": {
   "name": "python"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
